/* 
 * @Author: jade
 * @Date:   2015-12-21 10:33:08
 * @Last Modified by:   jade
 * @Last Modified time: 2016-01-17 21:15:03
 */
var Crawler = require('crawler');
var fs = require('fs');
var path = require('path');
var argv = require('yargs').argv;
var city = argv.city;
//node lianjia.js --city sh
//sh:上海，
var rootPath = 'http://' + city + '.lianjia.com';

var cityMap = {
    sh: {
        area: ['jingan', 'xuhui', 'huangpu', 'changning', 'putuo', 'baoshan', 'zhabei', 'hongkou', 'yangpu', 'minhang', 'jinshan', 'jiading', 'chongming', 'fengxian', 'songjiang', 'qingpu', 'shanghaizhoubian'],
        bk: ['pudongxinqu', 'beicai', 'biyun', 'caolu', 'chuansha', 'datuanzhen', 'geqing', 'gaohang', 'gaodong', 'huamu', 'hangtou', 'huinan', 'jinqiao', 'jinyang', 'kangqiao', 'lujiazui', 'laogangzhen', 'lingangxincheng', 'lianyang', 'nichengzhen', 'nanmatou', 'sanlin', 'shibo', 'shuyuanzhen', 'tangqiao', 'tangzhen', 'waigaoqiao', 'wanxiangzhen', 'weifang', 'xuanqiao', 'xinchang', 'yuqiao1', 'yangdong', 'yuanshen', 'yangjing', 'zhangjiang', 'zhuqiao', 'zhoupu']
    },
    bj: {
        area: ['dongcheng', 'xicheng', 'chaoyang', 'haidian', 'fengtai', 'shijingshan', 'tongzhou', 'changping', 'daxing', 'yizhuangkaifaqu', 'shunyi', 'fangshan', 'mentougou', 'pinggu', 'huairou', 'miyun', 'yanqing', 'yanjiao'],
        time: 60
    },
    sz: {
        area: ['luohu', 'futian', 'nanshan', 'yantian', 'baoan', 'longgang', 'longhuaxinqu', 'guangmingxinqu', 'pingshanxinqu', 'dapengxinqu']
    },
    tj: {
        area: ['heping', 'nankai', 'hexi', 'hebei', 'hedong', 'hongqiao', 'xiqing', 'beichen', 'dongli', 'jinnan', 'tanggu', 'kaifaqutj']
    },
    hz: {
        area: ['jiande', 'xihu', 'xiacheng', 'jianggan', 'gongshu', 'shangcheng', 'binjiang', 'yuhang', 'xiaoshan', 'linan']
    },
    wh: {
        area: ['jiangan', 'jianghan', 'qiaokou', 'dongxihu', 'wuchang', 'qingshan', 'hongshan', 'hanyang']
    },
    cd: {
        area: ['jinjiang', 'qingyang', 'wuhou', 'gaoxinnan', 'chenghua', 'jinniu', 'tianfuxinqu', 'shuangliu', 'wenjiang', 'pixian', 'gaoxinxiqu', 'longanyi', 'xindou']
    },
    cq: {
        area: ['jiangbei', 'yubei', 'nanan', 'banan', 'shapingba', 'jiulongpo', 'yuzhong', 'dadukou', 'jiangjing', 'qijiang', 'changshou1', 'fuling']
    },
    nj: {
        area: ['gulou', 'jianye', 'qinhuai', 'xuanwu', 'yuhuatai', 'qixia', 'jiangning', 'pukou', 'liuhe', 'lishui', 'gaochun']
    },
}

function store(str, fileName) {
    fs.appendFile(fileName || './chengjiao/' + city + '.txt', str + '\n', function(err) {
        err && console.log(err)
    });
}


function getRandom() {
    return ~~(Math.random() * 10) + 0.1
}

function processStr(str) {
    return str.replace(/,/g, '，').replace(/\n/g, '');
}

// if (page == 1) {
//     var tilte = 'IW小区ID,区域板块,小区名,户型,面积（单位：平米）,楼层,签约日期,成交价,成交单价'
//     store(tilte);
// }

function addQ(url, timer, cb, text) {
    (function(url, timer, cb, text) {
        setTimeout(function() {
            //console.log(text);
            c.queue(url);
            cb && cb();
        }, timer)
    })(url, timer, cb, text)
}
var index = 0;
var c = new Crawler({
    maxConnections: 200,
    skipDuplicates: true,
    // This will be called for each crawled page
    callback: function(error, result, $) {
        var url = result.uri;
        if (error) {
            console.log('-----------出错了：');
            //console.log(error);
            //c.queue(url);
            return false;
        }
        if (typeof $ != 'function') {
            console.log('-----------出错了1：');
            //c.queue(url);
            return false;
        }
        var url = result.uri;
        var search = url.replace(/.*quyu=(.+)&bankuai=(.+)/, '$1,$2');
        var quyu = search.split(',')[0];
        var bankuai = search.split(',')[1];

        if (url.indexOf('?xiaoqu') > 0) {
            //小区列表页面，先拿到总页数
            var links = $('.house-lst-page-box');
            var pages = 1;

            try {
                pages = JSON.parse(links.attr('page-data')).totalPage;
            } catch (e) {

            }
            for (var i = 0; i < pages; i++) {
                var urlS = url.split('/');

                var id = urlS[urlS.length - 2]
                var nurl = rootPath + '/xiaoqu/' + id + '/pg' + (i + 1) + '/?liebiao';
                //var timer = getRandom() * 1000 * 60 * 6 * (i + 1);
                var timer = 0;
                if (city == 'bj') {
                    timer = 1000 * 60 * 8 * i;
                } else {
                    timer = 1000 * 60 * 4 * i;
                }
                var logStr = '小区信息页数:' + timer / 1000 + 's后开始爬第' + (i + 1) + '页小区：'
                console.log(logStr);
                addQ(nurl, timer, function() {}, logStr)
            }
        } else {
            if (url.indexOf('?liebiao') > 0) {
                //小区列表
                $('#house-lst li .info-panel h2 a').each(function(itemIndex, item) {
                    var href = $(item).attr('href');
                    var id = href.split('/')[2]; //拿出来小区id

                    var quyu = $(item).closest('.info-panel').find('.con a').first().text();
                    var bankuai = $(item).closest('.info-panel').find('.con a').first().next().text();

                    var nurl = rootPath + '/chengjiao/c' + id + '/?quyu=' + quyu + '&bankuai=' + bankuai;
                    var timer = 0;
                    if (city == 'bj') {
                        timer = 1000 * 60 * 4 * itemIndex;
                    } else {
                        timer = 1000 * 60 * itemIndex;
                    }
                    
                    //var timer = getRandom() * 1000 * (itemIndex + 1);
                    index++;
                    var logStr = '小区信息详情:' + timer / 1000 + 's后开始爬第' + index + '条小区：';
                    console.log(logStr);
                    addQ(nurl, timer, function() {}, logStr)
                })
            } else {
                if (url.indexOf(rootPath + '/chengjiao/c')) {
                    //分析具体数据
                    $('.clinch-list .info-panel').each(function(itemIndex, item) {
                        console.log(url)
                        var str = quyu + ' ' + bankuai;
                        var infoDom = $(item).find('h2 a');
                        var info = infoDom.text().split(' ');
                        if (infoDom.length == 0) {
                            infoDom = $(item).find('h2 div');

                            info = infoDom.text().replace('历史成交，暂无详情页', '').split(' ');
                        }
                        str += ',' + info[0] + ',' + info[1] + ',' + info[2];
                        str += ',' + $(item).find('.con').text();
                        var info = $(item).find('.div-cun');
                        var date = $(info[0]).text();
                        str += ',' + date;
                        str += ',' + $(info[2]).text();
                        str += ',' + $(info[1]).text();

                        console.log('成交记录：:' + str + ',' + url);

                        var year = ~~date.split('.')[0];


                        if (city == 'bj') {
                            if (year >= 2012) {
                                store(str);
                            }
                        } else {
                            store(processStr(str));
                        }
                    })
                } else {
                    //先把所有的页数拿到
                    var links = $('.house-lst-page-box');
                    var pages = 1;

                    try {
                        pages = JSON.parse(links.attr('page-data')).totalPage;
                    } catch (e) {

                    }
                    for (var i = 0; i < pages; i++) {
                        var urlS = url.split('/');

                        var id = urlS[urlS.length - 2]
                        var nurl = rootPath + '/chengjiao/pg' + (i + 1) + id + '/?quyu=' + quyu + '&bankuai=' + bankuai;
                        //var timer = getRandom() * 1000 * 30 * (i + 1);
                        var timer = 0;
                        if (city == 'bj') {
                            timer = 1000 * 30 * i;
                        } else {
                            timer = 1000 * 10 * i;
                        }
                        var logStr = '成交记录分页:' + timer / 1000 + 's后开始爬第' + (i + 1) + '页成交记录：';
                        console.log(logStr);
                        addQ(nurl, timer, function() {}, logStr)
                    }
                }
            }
        }
    }
});

var data = cityMap[city];
var time = data.time;
data = data.area.concat(data.bk || []);
for (var ai = 0, alen = data.length; ai < alen; ai++) {
    var area = data[ai];
    var timer = 1000 * 60 * (time || 30) * (ai % 6);
    var logStr = '区域信息：' + timer / 1000 + 's后开始爬：' + area;
    console.log(logStr);
    var url = rootPath + '/xiaoqu/' + area + '/?xiaoqu';
    console.log(url);
    addQ(url, timer, function() {}, logStr)
}
